#TODO make it for the year from 2009 to 2011
#currently, NO USE.

outputFeatureFileName = "/data1/team/weijiang/machine-learning-project-related/trec-related/trec2011/priorityWarcDocumentsIDs_2011_sorted.txt"
print "generate the set of documents I need to crawl first."

fileList = []

'''
#info at position 1
#2009
fileList.append( ("/data1/team/weijiang/machine-learning-project-related/trec-related/trec2009/prels.1-50",1,2009) )
fileList.append(  ("/data1/team/weijiang/machine-learning-project-related/trec-related/trec2009/prels.catA.1-50",1,2009) )
fileList.append( ("/data1/team/weijiang/machine-learning-project-related/trec-related/trec2009/prels.catB.1-50",1,2009) )


#info at position 2
#2009
fileList.append( ("/data1/team/weijiang/machine-learning-project-related/trec-related/trec2009/qrels.diversity",2,2009))

#2010
fileList.append( ("/data1/team/weijiang/machine-learning-project-related/trec-related/trec2010/10.adhoc-qrels.final",2,2010))
fileList.append( ("/data1/team/weijiang/machine-learning-project-related/trec-related/trec2010/10.diversity-qrels.final",2,2010))
'''

#2011
fileList.append( ("/data1/team/weijiang/machine-learning-project-related/trec-related/trec2011/qrels.adhoc",2,2011))

warcDocumentIDFreqsDict = {}


for fileNameTuple in fileList:
    (fileName,position,year) = fileNameTuple
    print "processing ",fileName
    inputFileHandler1 = open(fileName,"r")
    for currentLine in inputFileHandler1.readlines():
        if position == 1:
            warcDocumentID = currentLine.split(" ")[1]
        elif position == 2:
            warcDocumentID = currentLine.split(" ")[2]
        
        if warcDocumentID not in warcDocumentIDFreqsDict:
            warcDocumentIDFreqsDict[warcDocumentID] = 0
        else:
            warcDocumentIDFreqsDict[warcDocumentID] += 1
    
    inputFileHandler1.close()


#clueweb09-en0000-79-12800
maxValue = -1
mostFreqWarcDocumentID = ""

outputQueryFeatureFileHandler = open(outputFeatureFileName,"w")
outputLineListForIDFreqs = []

for currentID in warcDocumentIDFreqsDict:
    outputLineListForIDFreqs.append( (currentID, warcDocumentIDFreqsDict[currentID]) )

outputLineListForIDFreqs.sort(cmp=None, key=None, reverse=False)
for currentElement in outputLineListForIDFreqs:
    (warcTrecID,freq) = currentElement
    outputLine = warcTrecID + " " + str(freq)
    outputQueryFeatureFileHandler.write(outputLine + "\n")
    if maxValue < freq:
        mostFreqWarcDocumentID, maxValue = warcTrecID,freq

outputQueryFeatureFileHandler.close()

print "General Info"
print "IDs Size:",len(warcDocumentIDFreqsDict)
print "mostFreqWarcDocumentID:",mostFreqWarcDocumentID
print "maxValue:",maxValue



    
    
    
    
    
    
    
